{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from lxml import etree\n",
    "# url = 'http://bj.ganji.com/ershoufang/'\n",
    "url = 'https://bj.58.com/ershoufang/'\n",
    "\n",
    "headers = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36',\n",
    "}\n",
    "response = requests.get(url, headers=headers)\n",
    "html = etree.HTML(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_file(content:str):\n",
    "    filename = '二手房数据.csv'\n",
    "    with open(filename, 'a+', encoding='gbk', errors='ignore') as file:\n",
    "        file.write(content)\n",
    "        file.write('\\n')\n",
    "        \n",
    "## w 覆盖写入\n",
    "## wb 二进制写入\n",
    "## a+ 追加写入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120\n"
     ]
    }
   ],
   "source": [
    "li_list = html.xpath('.//li[@class=\"sendsoj\"]')\n",
    "print(len(li_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['永清高端大盘校区房首期开盘价6800起可落护临高\\xa0'] ['62.5'] ['万'] ['7353元/㎡'] ['85.0㎡\\xa0'] ['2室2厅1卫', '\\n                                                                                    ', '\\n                                        \\n                                         -                                                                                     ', '\\n                                                                                 -                                                                                     ', '\\n                                                                            ']\n"
     ]
    },
    {
     "ename": "UnicodeDecodeError",
     "evalue": "'gbk' codec can't decode byte 0xab in position 10: illegal multibyte sequence",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-26-cfd8843d7e75>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m     \u001b[0minfo_str\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'{},{},{},{},{}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtitle\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmoney\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munit\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marea\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhuxing\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 20\u001b[1;33m     \u001b[0minfo_str\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minfo_str\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'gbk'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     21\u001b[0m     \u001b[0mwrite_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minfo_str\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m: 'gbk' codec can't decode byte 0xab in position 10: illegal multibyte sequence"
     ]
    }
   ],
   "source": [
    "for li_item in li_list:\n",
    "    title = li_item.xpath('.//div[@class=\"list-info\"]/h2[@class=\"title\"]/a/text()')\n",
    "    title = [t.replace(',', '，') for t in title]\n",
    "    price = li_item.xpath('.//div[@class=\"price\"]/p[@class=\"sum\"]/b/text()')\n",
    "    wan = li_item.xpath('.//div[@class=\"price\"]/p[@class=\"sum\"]/text()')\n",
    "    money = price[0] + wan[0]\n",
    "    unit = li_item.xpath('.//div[@class=\"price\"]/p[@class=\"unit\"]/text()')\n",
    "    area = li_item.xpath('.//div[@class=\"list-info\"]/p[@class=\"baseinfo\"]/span[2]/text()')\n",
    "    huxing = li_item.xpath('.//div[@class=\"list-info\"]/p[@class=\"baseinfo\"]/span[1]/text()')\n",
    "    print(title, price, wan, unit, area, huxing)\n",
    "    \n",
    "#     info = []\n",
    "#     info.extend(title)\n",
    "#     info.extend(price)\n",
    "#     info.extend(money)\n",
    "#     info.extend(area)\n",
    "#     info.extend(huxing)\n",
    "    \n",
    "    info_str = '{},{},{},{},{}'.format(title[0], money, unit[0], area[0], huxing[0])\n",
    "    write_file(info_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
